import scrapy

from novelBooks.items import NovelbooksItem


class BooksSpider(scrapy.Spider):
    name = 'books'
    allowed_domains = ['www.zwdu.com']
    start_urls = ['https://www.zwdu.com']
    # 一级页面
    def parse(self, response):
        type_list = response.xpath('//div[@class="nav"]/ul/li')
        for val in type_list[1:2]:
            item = NovelbooksItem()
            url = self.start_urls[0] + val.xpath('./a/@href').get()
            typeName = val.xpath('./a/text()').get()
            item['typeName'] = typeName
            yield scrapy.Request(
                url=url,
                meta={'item': item},
                callback=self.two_parse
            )
    # 二级页面  书列表
    def two_parse(self, response):
        item = response.meta['item']
        books_list = response.xpath('//div[@id="newscontent"]/div/ul/li')
        for val in books_list[0:2]:
            bookName = val.xpath('./span[1]/a/text()').get()
            url = self.start_urls[0] + val.xpath('./span[1]/a/@href').get()
            yield scrapy.Request(
                url=url,
                meta={'item': item, 'bookName': bookName},
                callback=self.three_parse
            )
    # 三级页面 章列表
    def three_parse(self, response):
        item = response.meta['item']
        bookName = response.meta['bookName']
        author = response.xpath('//div[@id="info"]/p[1]/text()').get().split('：')[1]
        time = response.xpath('//div[@id="info"]/p[3]/text()').get().split('：')[1]
        status = response.xpath('//div[@id="info"]/p[2]/text()').get().split('：')[1].split(',')[0]
        description = response.xpath('//div[@id="intro"]/p[1]/text()').get()
        title_list = response.xpath('//div[@id="list"]/dl/dd')
        for val in title_list[0:1]:
            titleName = val.xpath('./a/text()').get()
            url = self.start_urls[0] + val.xpath('./a/@href').get()
            yield scrapy.Request(
                url=url,
                meta={
                    'item': item,
                    'author': author,
                    'time': time,
                    'status': status,
                    'description': description,
                    'bookName': bookName,
                    'titleName': titleName},
                callback=self.four_parse
            )

    # 四级页面 章内容
    def four_parse(self, response):
        item = response.meta['item']
        item['bookName'] = response.meta['bookName']
        item['title'] = response.meta['titleName']
        item['author'] = response.meta['author']
        item['time'] = response.meta['time']
        item['status'] = response.meta['status']
        item['description'] = response.meta['description']
        con = response.xpath('//div[@id="content"]/text()').extract()
        content = '\n'.join(con)
        item['content'] = content
        yield item